{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/limeng/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载。\n",
    "\n",
    "<font color=#ff0000>**这里将data_dir改为适合你的运行环境的目录**</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-5243832fc397>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/limeng/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/limeng/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /home/limeng/TensorFlow-Examples/input_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/limeng/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /home/limeng/TensorFlow-Examples/input_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/limeng/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting /home/limeng/TensorFlow-Examples/input_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /home/limeng/TensorFlow-Examples/input_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/limeng/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = '/home/limeng/TensorFlow-Examples/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一个非常非常简陋的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the model-2层\n",
    "x = tf.placeholder(tf.float32, [None, 784], name = 'X')\n",
    "#参数初始化\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 300], stddev=0.1))\n",
    "b1 = tf.Variable(tf.constant(0.1, shape=[300]))\n",
    "W2 = tf.Variable(tf.truncated_normal([300, 200], stddev=0.1))\n",
    "b2 = tf.Variable(tf.constant(0.1, shape=[200]))\n",
    "W3 = tf.Variable(tf.truncated_normal([200, 10], stddev=0.1))\n",
    "b3 = tf.Variable(tf.constant(0.1, shape=[10]))\n",
    "#relu激活函数\n",
    "layer1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "layer2 = tf.nn.relu(tf.matmul(layer1, W2) + b2)\n",
    "\n",
    "y = tf.matmul(layer2, W3) + b3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义我们的ground truth 占位符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10], name = 'Y')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来我们计算交叉熵，注意这里不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "另一个注意点就是，softmax_cross_entropy_with_logits的logits参数是**未经激活的wx+b**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "#加正则项\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "regularizer = tf.contrib.layers.l2_regularizer(0.001)\n",
    "regularization = regularizer(W1) + regularizer(W2)\n",
    "loss = cross_entropy + regularization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成一个训练step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#decay 学习率\n",
    "LEARNING_RATE_BASE = 0.1 # 设置初始学习率为0.1\n",
    "LEARNING_RATE_DECAY = 0.99 # 设置学习衰减率为0.99\n",
    "LEARNING_RATE_STEP = mnist.train.num_examples / 100 # 设置喂入多少轮BATCH_SIZE之后更新一次学习率,一般设置为 总样本数/BATCH_SIZE\n",
    "global_step = tf.Variable(0,trainable = False)\n",
    "\n",
    "learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,\n",
    "                                           global_step,\n",
    "                                           LEARNING_RATE_STEP,\n",
    "                                           LEARNING_RATE_DECAY,\n",
    "                                           staircase=True)\n",
    "\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在这里我们仍然调用系统提供的读取数据，为我们取得一个batch。\n",
    "然后我们运行3k个step(5 epochs)，对权重进行优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After 0 training step(s),acc is 0.107\n",
      "After 500 training step(s),acc is 0.9377\n",
      "After 1000 training step(s),acc is 0.9532\n",
      "After 1500 training step(s),acc is 0.963\n",
      "After 2000 training step(s),acc is 0.9651\n",
      "After 2500 training step(s),acc is 0.9676\n",
      "After 3000 training step(s),acc is 0.9704\n",
      "After 3500 training step(s),acc is 0.9735\n",
      "After 4000 training step(s),acc is 0.9746\n",
      "After 4500 training step(s),acc is 0.9728\n",
      "After 5000 training step(s),acc is 0.975\n",
      "After 5500 training step(s),acc is 0.9786\n",
      "After 6000 training step(s),acc is 0.9775\n",
      "After 6500 training step(s),acc is 0.9789\n",
      "After 7000 training step(s),acc is 0.9765\n",
      "After 7500 training step(s),acc is 0.9767\n",
      "After 8000 training step(s),acc is 0.9784\n",
      "After 8500 training step(s),acc is 0.9778\n",
      "After 9000 training step(s),acc is 0.9776\n",
      "After 9500 training step(s),acc is 0.9784\n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "for _ in range(10000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    if _ % 500 == 0:\n",
    "        test_acc = sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels})\n",
    "        print(\"After %d training step(s),acc is %g\"%(_, test_acc))\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "验证我们模型在测试数据上的准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_acc is 0.9808\n"
     ]
    }
   ],
   "source": [
    "  # Test trained model\n",
    "print(\"test_acc is %g\"%sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到92%左右的准确率。\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至98%以上的准确率。\n",
    "Hint：\n",
    "- 多隐层\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 隐层神经元数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
